import numpy
import os
import random
import feedparser
import operator

def loadDataSet():
    postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
                   ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
                   ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
                   ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
                   ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
                   ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']
                   ]
    classVec = [0,1,0,1,0,1]
    return postingList, classVec


def createVocabList(dataSet):       # define a vocabulary
    vocabSet = set([])              # define a empty set by a empty list
    for document in dataSet:        # every document is a sentence from dataset
        vocabSet = vocabSet | set(document)     # every word from sentences will only find once in the vocabSet
    return list(vocabSet)


def setOfWords2Vec(vocabList, inputSet):
    retVec = [0] * len(vocabList)        # length of Vocabulary
    for word in inputSet:
        if word in vocabList:               # the word is found from vocabulary
            id = vocabList.index(word)
            retVec[id] = 1
        else:
            pass    # print("the word: %s is not in my Vocabulary !" % word)
    return retVec

def bagOfWords2Vec(vocabList, inputSet):
    retVec = [0] * len(vocabList)        # length of Vocabulary
    for word in inputSet:
        if word in vocabList:               # the word is found from vocabulary
            id = vocabList.index(word)
            retVec[id] += 1
        else:
            pass    # print("the word: %s is not in my Vocabulary !" % word)
    return retVec

def createTrainMatrix(postingList, vocabList):
    trainMat = []
    for onePost in postingList:
        trainMat.append(setOfWords2Vec(vocabList, onePost))     # one item of trainMat is a vacabulary Vec of a post
    return trainMat

def trainNB0(trainMatrix, trainCategory):                      # one item of trainMat is a vacabulary Vec of a post
     numTrainDocs = len(trainMatrix)
     numWords = len(trainMatrix[0])
     pAbusive = sum(trainCategory)/float(numTrainDocs)          # P(C1)
     oneWordCountVec0 = numpy.ones(numWords)                   # P(W|C0) = oneWordCountVec0 / allWordCount0
     oneWordCountVec1 = numpy.ones(numWords)                   # P(W|C1) = oneWordCountVec1 / allWordCount1
     allWordCount0 = 2.0
     allWordCount1 = 2.0
     for i in range(numTrainDocs):
         if trainCategory[i] == 0:
             oneWordCountVec0 += trainMatrix[i]
             allWordCount0 += sum(trainMatrix[i])
         else:
             oneWordCountVec1 += trainMatrix[i]
             allWordCount1 += sum(trainMatrix[i])
     p0Vect = numpy.log(oneWordCountVec0 / allWordCount0)
     p1Vect = numpy.log(oneWordCountVec1 / allWordCount1)
     return  p0Vect, p1Vect, pAbusive

def classifyByNB(inputPostVec, p0Vect, p1Vect, pAbusive):
    p0 = sum(inputPostVec * p0Vect) + numpy.log(pAbusive)
    p1 = sum(inputPostVec * p1Vect) + numpy.log(pAbusive)
    #print("P0 = %d"%p0)
    #print("P1 = %d"%p1)
    if p0 > p1:
        return 0
    else:
        return 1

def testNB():
    trainDataSet, classVec = loadDataSet()
    vocabList = createVocabList(trainDataSet)
    trainMat = createTrainMatrix(trainDataSet, vocabList)
    p0Vect, p1Vect, pAbusive = trainNB0(trainMat, classVec)
    testPost = ["May", "I", "help", "you", "for", "this", "question"]
    testPostVec = numpy.array(setOfWords2Vec(vocabList, testPost))
    testRs = classifyByNB(testPostVec, p0Vect, p1Vect, pAbusive)
    print(testRs)
    testPost = ["He", "he", "stupid"]
    testPostVec = numpy.array(setOfWords2Vec(vocabList, testPost))
    testRs = classifyByNB(testPostVec, p0Vect, p1Vect, pAbusive)
    print(testRs)

##### filte spam email #####

def loadEmail(filePath):     # return a email 2D-list with word splited
    emailData = []
    fileList = os.listdir(filePath)
    for oneFile in fileList:
        fileName = filePath + '/' + oneFile
        emailData.append(open(fileName).read())
    return emailData

def textParse(bigString):
    import re
    listOfTokens = re.split(r'\W*', bigString)
    return [tok.lower() for tok in listOfTokens if len(tok) > 2]


def testSpamEmail():
    emailData = []
    emailClass = []
    allWord = []
    data = loadEmail("./email/ham")
    for oneEmail in data:
        wordList = textParse(oneEmail)
        emailData.append(wordList)
        allWord.extend(wordList)
        emailClass.append(0)
    data = loadEmail("./email/spam")
    for oneEmail in data:
        wordList = textParse(oneEmail)
        emailData.append(wordList)
        allWord.extend(wordList)
        emailClass.append(1)
    vocabList = createVocabList(emailData)
    testData = []
    testClass = []
    for i in range(10):     # 选10个用于后面测试
        randIndex = int(random.uniform(0, len(emailData)))
        testData.append(emailData[randIndex])
        testClass.append(emailClass[randIndex])
        del(emailData[randIndex])
        del(emailClass[randIndex])

    # build a train matrix
    emailTrainMat = createTrainMatrix(emailData, vocabList)
    # get bayes parameters
    p0, p1, pSpam = trainNB0(emailTrainMat, emailClass)
    # test the classify
    errorCount = 0.0
    for index, testEmail in enumerate(testData):
        wordVec = setOfWords2Vec(vocabList, testEmail)
        ret = classifyByNB(numpy.array(wordVec), p0, p1, pSpam)
        if ret != testClass[index]:
            errorCount += 1
    print("the error rate is %f" %(errorCount/len(testClass)))

##### Get the most frequent word in a class (use RSS data) #####
def calcMostFraq(vocabList, fullText):
    frqDict = {}
    for token in vocabList:
        frqDict[token] = fullText.count(token)
    sortedFrqDict = sorted(frqDict.items(), key=operator.itemgetter(1), reverse=True)
    return sortedFrqDict[:50]

def localWords(feed0, feed1):
    docList = []
    classList = []
    fullText = []
    minLen = min(len(feed1['entries']), len(feed0['entries']))
    for i in range(minLen):
        wordList = textParse(feed1['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        wordList = textParse(feed0['entries'][i]['summary'])
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)
    vocabList = createVocabList(docList)
    top30Words = calcMostFraq(vocabList, fullText)      # return a dict
    for pairW in top30Words:
        # if pairW[0] not in vocabList:
        #     print("Except:not word find in vocablist")
        vocabList.remove(pairW[0])             # remove the frequent word just like 'I am ...'
        print("remove frequent word %s"%pairW[0])

    # choice data to test
    testSet = []
    testClass = []
    for i in range(20):                     # random choice 20 word to test
        randIndex = int(random.uniform(0, len(docList)))
        testSet.append(docList[randIndex])
        testClass.append(classList[randIndex])
        del(docList[randIndex])
        del(classList[randIndex])

    #build train matrix
    docTrainMat = createTrainMatrix(docList, vocabList)
    # get bayes parameters
    pWordInFeed0, pWordInFeed1, pFeed = trainNB0(docTrainMat, classList)
    # test the classify
    errorCount = 0.0
    for index, testWord in enumerate(testSet):
        wordVec = setOfWords2Vec(vocabList, testWord)
        ret = classifyByNB(numpy.array(wordVec), pWordInFeed0, pWordInFeed1, pFeed)
        if ret != testClass[index]:
            errorCount += 1
    print("the error rate is %f" %(errorCount/len(testClass)))
    return vocabList, pWordInFeed0, pWordInFeed1, pFeed

def getTopWords(ny, sf):
    vocabList, p0, p1, pFeed = localWords(ny, sf)
    topNY = []
    topSF = []
    for i in range(len(p0)):
        if p0[i] > -4.5: topNY.append((vocabList[i], p0[i]))
        if p1[i] > -4.5: topSF.append((vocabList[i], p1[i]))
    sortedTopNY = sorted(topNY, key=operator.itemgetter(1), reverse=True)
    sortedTopSF = sorted(topSF, key=operator.itemgetter(1), reverse=True)
    print("------------Top words of NY------------")
    for oneWord in sortedTopNY:
        print(oneWord[0], oneWord[1])
    print("------------Top words of SF------------")
    for oneWord in sortedTopSF:
        print(oneWord[0], oneWord[1])

def getFeed():
    ny = feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
    sf = feedparser.parse('http://sfbay.craigslist.org/stp/index.rss')
    return ny, sf


if __name__ == "__main__":
    ny, sf = getFeed()
    vl, p0, p1, pf = localWords(ny, sf)
    getTopWords(ny, sf)

    #print(vl)
    # print(p0)
    # print(p1)
    # print(pf)
